require 'rubygems'
require 'statsample'
require 'pathname'
require './debug.rb'
require 'reporter'


module Mkuz

    class Analyzer
        
        @@MAX_NUMBER_OF_WORDS           = 70
        @@NUMBER_OF_APPS_TO_FIND_WORDS  = 10000
        @@STORE_PROCESSED_ITEMS         = true
        
        def initialize(path)
            raise "data path is empty" if path.empty?
            @data_path = path
            
            Dir::mkdir(images_dir_abs) if not File::exists?(images_dir_abs)
            Dir::mkdir(word_images_dir_abs) if not File::exists?(word_images_dir_abs)
        end
        
        ### Paths ###
        
        def data_path
            @data_path
        end
        def data_filename
            Pathname.new(@data_path).basename.to_s
        end
        def processed_data_path
            path = data_path.gsub(".csv", ".processed.csv")
            raise if path == data_path
            return path
        end
        def images_dir_abs
            "#{@data_path}.d/"
        end
        def images_dir_rel
            Pathname.new(images_dir_abs).basename.to_s + '/'
        end
        def word_images_dir_abs
            "#{images_dir_abs}words/"
        end
        def word_images_dir_rel
            "#{images_dir_rel}words/"
        end
        def report_path_abs
            "#{data_path}.html"
        end
        
        ### Starts analysing ###
        
        def run
            time_elapsed("analazing data from #{data_path}"){
                load_file()
                process_items();
                extract_words()
                store_items()
                analyze_items()
            }
        end
        
        def analyze_items()
            time_elapsed("analazing items"){
                pop_items = []
                @items.each{ |item| pop_items << item if item['popularity'] > 0 }
            

                alpha = 0.05
                n = pop_items.size
                cv = spearman_cv(n, alpha)
            
                title = "Анализ данных из #{data_filename}"
                reporter = Reporter.new(report_path_abs, title)
                
                reporter << 
"Выделенная желтым зависимость существует с вероятностью не менее #{1-alpha} .
Т.е. вероятность, что наличие зависимости ошибочно, не более #{alpha}. " << 
"Коэф. Спирмена r<sub>s</sub> изменяется от -1 до 1, где 0 - нет связи, 1 - сильная прямая связь, -1 - сильная обратная связь. " <<
"<small>Последнее точное критичное значение коэф. Спирмена cv для n=100 и alpha=0.05 равно 0.197. Для n > 100 используется приближенная формула 1.96/sqrt(n-1). </small>"

                reporter << { "section" => dependency_popularity_size(pop_items, alpha)}
                reporter << { "section" => dependency_popularity_rated(pop_items, alpha)}
                reporter << { "section" => dependency_stars_size(pop_items, alpha)}
                reporter << { "section" => dependency_stars_rated(pop_items, alpha)}
                dependency_popularity_word(pop_items, alpha, reporter)

                reporter.close
            }
        end
        
        def dependency_popularity_word(items, alpha, reporter)
            
            n = items.size
            cv = spearman_cv(n, alpha)
            
            reporter << "<h3>Зависимость популярности от слов в первом абзаце</h3>"
            reporter << "Всего слов: #{@word_columns.size}"
            
            plots = []
            (0..@word_columns.length-1).each{ |i|
                #break if i > 2 # TODO 
                word = @word_columns[i]
                puts "analysing word '#{word}' (#{i+1}/#{@word_columns.length})..."
                
                a1 = []
                a2 = []
                (0..items.length-1).each{ |j| 
                    a1 << items[j]['popularity']; 
                    a2 << items[j]['words'][i] 
                }
                
                r_s = Statsample::Bivariate.spearman(a1.to_scale, a2.to_scale)
                r_s = 0 if r_s.nan?
                
                # add to report
                plot = []
                
                if make_decision_spearman(r_s, n, alpha) then
                    plot << "<h3 style='background:yellow'>#{word}</h3>\n" 
                else
                    plot << "<h3>#{word}</h3>\n"
                end
                plot_id = "popularity_word_#{word}"
                
                plot << "r<sub>s</sub> = #{r_s}, cv = #{cv}, df = #{n}, alpha = #{alpha}"
                plot << {"scatterplot" => { 
                        'dir_rel'   => word_images_dir_rel,
                        'dir_abs'   => word_images_dir_abs,
                        'x_samples' => a2,
                        'y_samples' => a1,
                        'x_desc'    => "'плотность' слова в первом абзаце (кол-во повторений / общее кол-во слов)",
                        'y_desc'    => "популярность программы (количество оценок / время)",
                        'id'        => plot_id
                    }
                }
                plots << [r_s, plot]
            }
            sorted = plots.sort_by{|pair| pair[0]}.reverse!
            sorted.each{|pair| reporter << pair[1]}
        end
        
        
        def dependency_stars_rated(items, alpha)
            
            n = items.size
            cv = spearman_cv(n, alpha)
            
            a1 = []
            a2 = []
            (0..items.length-1).each {|j| 
                
                s5 = items[j]['5 stars']
                s4 = items[j]['4 stars']
                s3 = items[j]['3 stars']
                s2 = items[j]['2 stars']
                s1 = items[j]['1 star']
                
                a1 << (5*s5 + 4*s4 + 3*s3 + 2*s2 + 1*s1)/(s5 + s4 + s3 + s2 + s1)
                a2 << items[j]['rated']   
            }

            r_s = Statsample::Bivariate.spearman(a1.to_scale, a2.to_scale)
            r_s = 0 if r_s.nan?
            
            content = []
            
            title = "Зависимость средней оценки от ограничения по возрасту"
            if make_decision_spearman(r_s, n, alpha) then
                content << "<h3 style='background:yellow'>#{title}</h3>\n" 
            else
                content << "<h3>#{title}</h3>\n"
            end
            
            content << "r<sub>s</sub> = #{r_s}, cv = #{cv}, df = #{n}, alpha = #{alpha}"
            
            content << {"scatterplot" => { 
                    'dir_rel'   => images_dir_rel,
                    'dir_abs'   => images_dir_abs,
                    'x_samples' => a2,
                    'y_samples' => a1,
                    'x_desc'    => "ограничение по возрасту",
                    'y_desc'    => "средняя оценка",
                    'id'        => "stars_rated"
                }
            }
            return content
        end

        def dependency_stars_size(items, alpha)
            
            n = items.size
            cv = spearman_cv(n, alpha)
            
            a1 = []
            a2 = []
            (0..items.length-1).each {|j| 
                
                s5 = items[j]['5 stars']
                s4 = items[j]['4 stars']
                s3 = items[j]['3 stars']
                s2 = items[j]['2 stars']
                s1 = items[j]['1 star']
                
                a1 << (5*s5 + 4*s4 + 3*s3 + 2*s2 + 1*s1)/(s5 + s4 + s3 + s2 + s1)
                a2 << items[j]['size']   
            }

            r_s = Statsample::Bivariate.spearman(a1.to_scale, a2.to_scale)
            r_s = 0 if r_s.nan?
            
            content = [] 
            
            title = "Зависимость средней оценки от размера программы"
            if make_decision_spearman(r_s, n, alpha) then
                content << "<h3 style='background:yellow'>#{title}</h3>" 
            else
                content << "<h3>#{title}</h3>"
            end
            
            content << "r<sub>s</sub> = #{r_s}, cv = #{cv}, df = #{n}, alpha = #{alpha}"
            
            content << {"scatterplot" => { 
                    'dir_rel'   => images_dir_rel,
                    'dir_abs'   => images_dir_abs,
                    'x_samples' => a2,
                    'y_samples' => a1,
                    'x_desc'    => "размер программы в Мб",
                    'y_desc'    => "средняя оценка",
                    'id'        => "stars_size"
                }
            }
            return content
        end


        def dependency_popularity_size(items, alpha)
            
            n = items.size
            cv = spearman_cv(n, alpha)
            

            a1 = []
            a2 = []
            (0..items.length-1).each {|j| a1 << items[j]['popularity']; a2 << items[j]['size'] }

            r_s = Statsample::Bivariate.spearman(a1.to_scale, a2.to_scale)
            r_s = 0 if r_s.nan?
            
            content = []
            
            title = "Зависимость популярности от размера программы"
            if make_decision_spearman(r_s, n, alpha) then
                content << "<h3 style='background:yellow'>#{title}</h3>" 
            else
                content << "<h3>#{title}</h3>"
            end
            
            content << "r<sub>s</sub> = #{r_s}, cv = #{cv}, df = #{n}, alpha = #{alpha}"
            
            content << {"scatterplot" => { 
                    'dir_rel'   => images_dir_rel,
                    'dir_abs'   => images_dir_abs,
                    'x_samples' => a2,
                    'y_samples' => a1,
                    'x_desc'    => "размер программы в Мб",
                    'y_desc'    => "популярность программы (количество оценок / время)",
                    'id'        => "popularity_size"
                }
            }
            return content
        end

        def dependency_popularity_rated(items, alpha)
            
            n = items.size
            cv = spearman_cv(n, alpha)
            
            a1 = []
            a2 = []
            (0..items.length-1).each{ |j| 
                a1 << items[j]['popularity']; 
                a2 << items[j]['rated'] 
            }
            
            r_s = Statsample::Bivariate.spearman(a1.to_scale, a2.to_scale)
            r_s = 0 if r_s.nan?
            
            content = []
            title = "Зависимость популярности от ограничения по возрасту"
            if make_decision_spearman(r_s, n, alpha) then
                content << "<h3 style='background:yellow'>#{title}</h3>\n" 
            else
                content << "<h3>#{title}</h3>\n"
            end
            
            content << "r<sub>s</sub> = #{r_s}, cv = #{cv}, df = #{n}, alpha = #{alpha}"
            
            content << {"scatterplot" => { 
                    'dir_rel'   => images_dir_rel,
                    'dir_abs'   => images_dir_abs,
                    'x_samples' => a2,
                    'y_samples' => a1,
                    'x_desc'    => "ограничение по возрасту",
                    'y_desc'    => "популярность программы (количество оценок / время)",
                    'id'        => "popularity_rated"
                }
            }
            return content
        end

        def test_compute_spearman_correlation()
            samples1 = [19, 15, 17, 18, 17, 18, 21, 21, 15, 13]
            samples2 = [19, 17, 17, 17, 17, 19, 20, 19, 15, 14]
            r_s = compute_spearman_correlation(samples1, samples2)
            puts "test_compute_spearman_correlation: expected 0.917, computed #{r_s}"
            
            ## THE SAME BY STATSAMPLE ##
            v1 = samples1.to_vector(:scale)
            v2 = samples2.to_vector(:scale)
            puts "spearman by statsample: #{Statsample::Bivariate.spearman(v1, v2)}"
        end
        
        def make_decision_spearman (r_s, n, alpha)
            # H0: p_s = 0, no correlation
            # H1: p_s != 0
            #alpha = 0.05 # probability to reject H0 by mistake 
            
            cv = spearman_cv(n, alpha) # df == n - 2 ??
            return r_s >= cv  # reject null hypothesis 
        end
        
        # two-tailed
        def spearman_cv(df, alpha)
            raise "alpha equals 0.05 is only supported" if alpha != 0.05
            raise "df is too small" if df < 6
            
            critical_values = { 
                6 => 0.886, 
                7 => 0.786,
                8 => 0.738,
                9 => 0.700,
                10 => 0.648,
                
                11 => 0.618,
                12 => 0.587,
                13 => 0.560,
                14 => 0.538,
                15 => 0.521,
                
                16 => 0.503,
                17 => 0.485,
                18 => 0.472,
                19 => 0.460,
                20 => 0.447,
                
                21 => 0.435,
                22 => 0.425,
                23 => 0.415,
                24 => 0.406,
                25 => 0.398,
                
                26 => 0.390,
                27 => 0.382,
                28 => 0.375,
                29 => 0.368,
                30 => 0.362,
                
                31 => 0.356,
                32 => 0.350,
                33 => 0.345, 
                34 => 0.340,
                35 => 0.335,
                36 => 0.330,
                37 => 0.325,
                38 => 0.321,
                39 => 0.317,
                40 => 0.313,
                
                45 => 0.294,
                50 => 0.279,
                55 => 0.266,
                60 => 0.255,
                65 => 0.244,
                70 => 0.235,
                75 => 0.227,
                80 => 0.220,
                85 => 0.213,
                90 => 0.207,
                95 => 0.202,
                100 => 0.197
            }
            
            if 6 <= df && df <= critical_values.keys.max then
                cv = critical_values.keys.sort.reverse.detect{|k| k <= df}
            elsif df > critical_values.keys.max then
                z = 1.96  # for alpha == 0.05
                cv = z / Math::sqrt(df-1)
                raise "bug" if cv > critical_values[critical_values.keys.max]
            else
                raise
            end
            puts "df=#{df}, alpha=#{alpha}, cv=#{cv}"
            return cv
        end
        
        def compute_spearman_correlation(samples1, samples2)
            raise "samples1.size != samples2.size" if samples1.size != samples2.size
            raise "samples1 and samples2 are empty" if samples1.size < 2
            
            ranks1 = compute_ranks(samples1)
            ranks2 = compute_ranks(samples2)
            
            diffs = []
            sorted1 = {}
            (0..samples1.length-1).each { |i| sorted1[i] = samples1[i] }
            sorted1 = sorted1.sort_by { |k,v| v}
            (0..sorted1.length-1).each { |i| 
                diffs << (ranks1[sorted1[i][1]][0] - ranks2[samples2[sorted1[i][0]]][0]) 
            }
            
            #test: diff = [0, 0.5, -2, 0, 0, 2, -1.5, 0, 1.5, -0.5]
            
            r_s = 0
            
            t_x = 0
            ranks1.each { |k,v| t_x += (v[1]**3 - v[1]) }
            t_x /= 12.0
            #puts "t_x = #{t_x}"
            
            t_y = 0
            ranks2.each { |k,v| t_y += (v[1]**3 - v[1]) }
            t_y /= 12.0
            #puts "t_y = #{t_y}"
            
            sum1 = 0
            diffs.each { |d| sum1 += d**2 }
            #puts "sum1 = #{sum1}"
            
            n = diffs.length
            #puts "n = #{n}"
            
            num = (n**3 - n)/6 - sum1 - t_x - t_y
            denom = Math::sqrt( ( (n**3 - n)/6 - 2*t_x ) * ( (n**3 - n)/6 - 2*t_y ) )
            r_s = num / denom
            
            return r_s
        end
        
        def compute_ranks(samples)
            ranks = {}
            
            sorted = samples.sort
            
            prev_sample = sorted[0]
            prev_rank = 1
            (1..sorted.length+1).each {|rank|
                if (rank > sorted.length || prev_sample != sorted[rank-1]) then
                    ranks[prev_sample] = [(prev_rank + rank - 1) / 2.0, rank - prev_rank]
                    prev_sample = sorted[rank-1] if rank <= sorted.length
                    prev_rank = rank
                end
            }
            #p ranks
            return ranks
        end
        
        def load_file()
            time_elapsed("loading file #{data_path}") {
                @items = []
                File.open(data_path, 'r'){ |file|
                    columns = []
                    file.each{ |line|
                        substrs = line.strip.split("\t")
                        if columns.empty? then
                            columns = substrs
                            next
                        end
                        
                        item = {}
                        (0..substrs.length-1).each { |i| item[columns[i]] = substrs[i] }
                        @items << item
                    }
                }
            }
        end
        
        def extract_words
            time_elapsed("extracting words"){
                @word_columns = []
                
                time_elapsed("finding most usable words"){
                    # begin with most popular apps
                    sorted_items = @items.sort_by{|item| item['popularity']}.reverse!
                    main_storage = {}
                    i = 0
                    sorted_items.each do |item|
                        break if (i+=1) > @@NUMBER_OF_APPS_TO_FIND_WORDS
                        extract_words_from_text(item["short_desc"], main_storage)
                    end
                    
                    #sort by density; begin with the most frequent ones
                    sorted_words = main_storage.sort_by { |k,v| v}.reverse!
                    sorted_words = sorted_words[0..@@MAX_NUMBER_OF_WORDS-1]
                    sorted_words.each { |pair| @word_columns << pair[0] }
                }
                
                time_elapsed("computing density of words"){
                    @items.each do |item|
                        storage = {}
                        extract_words_from_text(item["short_desc"], storage)

                        item_words = []
                        (0..@word_columns.length-1).each{ |i|
                            density = storage[@word_columns[i]]
                            density = 0 if density.nil?
                            item_words[i] = density
                        }
                        item['words'] = item_words
                    end
                }
            }
        end

        def extract_words_from_text(text, storage)
            words_count = text.split(/\s+/).size
            
            text.scan(/[0-9a-zA-Z\-]{3,}/){ |m|
                key = m.downcase
                density = storage[key]
                density = 0 if density.nil?
                storage[key] = density + 1.0 / words_count
            }
        end
        
        def process_items()
            time_elapsed("processing items"){
                i = 0
                @items.each do |item|
                    #time_elapsed("processing item #{item['name']} (#{i+=1} of #{@items.size})"){
                        process_item(item)
                    #}
                end
            }
        end
        
        def process_item(item)

            #parse size
            size_parts = item['size'].split(" ")
            item['size'] = size_parts[0].to_f if size_parts[1] == 'MB'
            raise "size units #{size_parts[1]} is not supported" if size_parts[1] != 'MB'
            raise "size must be positive" if item['size'] < 0
            
            # age rating e.g. Rated 12+
            item['rated'] = item['rated'].scan(/\d+/)[0].to_i
            raise unless item['rated'] >= 0 && item['rated'] < 50
            
            # stars since release date
            item['5 stars'] = item['5 stars'].to_i
            item['4 stars'] = item['4 stars'].to_i
            item['3 stars'] = item['3 stars'].to_i
            item['2 stars'] = item['2 stars'].to_i
            item['1 star'] = item['1 star'].to_i
            
            # estimate an amount of users that downloads the app since release date
            item['downloads'] = item['5 stars'] + item['4 stars'] + item['3 stars'] + item['2 stars'] + item['1 star'] 
            
            # release date
            release_date = Date.strptime(item['release_date'], '%b %d, %Y')
            current_date_str = data_filename.scan(/_(\d{4}-\d{2}-\d{2})\./)[0].to_s
            raise "filename #{data_filename} must end with date when the data was collected: '_yyyy-mm-dd.'" if current_date_str.empty?
            current_date = Date.strptime(current_date_str, '%Y-%m-%d')
            
            # period of time between current and release date in days
            item['period'] = (current_date - release_date).to_f
            raise if item['period'] < 0
            
            # number of downloads  per day
            item['popularity'] = 0 if item['period'] == 0
            item['popularity'] = (item['downloads'] / item['period']) if item['period'] != 0
            
        end
        
        
        def store_items()
            return unless @@STORE_PROCESSED_ITEMS
            
            time_elapsed("storing to file #{processed_data_path}"){
                File.open(processed_data_path, 'w'){ |file|
                    # header
                    header = "id\tname\tsize\trated\t5 stars\t4 stars\t3 stars\t2 stars\t1 star\tlink\trelease_date\tperiod\tdownloads\tpopularity"
                    columns = header.split(/\t/)
                    @word_columns.each {|col| header << "\t\"#{col}\"" }
                    header << "\n"
                    file.write(header)

                    # content
                    i = 0
                    @items.each{ |item|
                        i += 1
                        
                        next if item['popularity'] == 0

                        #puts "converting item #{i} of #{@items.size}..."
                        item_str = convert_item(item, columns) + "\n"
                        file.write(item_str)
                    }
                }
            }
        end

        def convert_item(item, columns)
            item_str = ""
            
            columns.each{ |col| item_str << "#{item[col]}\t" }
            item_str.strip!
            
            item['words'].each { |density| item_str << "\t#{density}" }
            
            return item_str
        end
        
    end
    
end #module
